package com.xiaotu.spider.maoyan.pageprocessor;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.xiaotu.common.model.CompanyBasicinfoModel;
import com.xiaotu.common.model.CompanyStarsModel;
import com.xiaotu.common.model.CompanyTvWorksModel;
import com.xiaotu.common.model.StarBasicinfoModel;
import com.xiaotu.common.model.StarTvWorksModel;
import com.xiaotu.common.util.CacheHandler;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.common.util.UUIDUtils;
import com.xiaotu.spider.AUrlPageProcessor;
import com.xiaotu.spider.maoyan.common.MaoYanDataSource;
import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;
@Component("MaoYanMovieStarConpanyPageProcessor")
public class MaoYanMovieStarConpanyPageProcessor extends AUrlPageProcessor {
	
	protected String resourceSiteId = Constants.SiteName.MAOYAN;//数据来源
	
	private String [] START_URL = {
			"http://piaofang.maoyan.com/dashboard"
	};
	
	@Autowired
	private CacheHandler cacheHandler;
	
    //全局去重艺人
    private static String SPIDER_DISTINCT_STAR_INFO = "SPIDER_DISTINCT_STAR_INFO";
    
    //全局去重公司
    private static String SPIDER_DISTINCT_COMPANY_INFO = "SPIDER_DISTINCT_COMPANY_INFO";
    
    //去重网大和电影中的艺人
    private static String STAR_INFO_SUPPLEMENT = "STAR_INFO_SUPPLEMENT";
    
    //去重网大和电影中的公司
    private static String COMPANY_INFO_SUPPLEMENT = "COMPANY_INFO_SUPPLEMENT";
    
	@Override
	public void process(Page page) {
		
		Map<String, Object> extrasMap = page.getRequest().getExtras();
		
		switch (this.getPageDataType(page))
        {
            case "/companylist":
	            this.basicinfoCompanylistProcess(page,extrasMap);
	            break;
            case "/celebritylist":
            	this.basicinfoPeoplelistProcess(page,extrasMap);
            	break;
            case "/celebrity?id":
            	this.starBasicinfoProcess(page,extrasMap);
            	break;
            case "/companydetail":
            	this.companyBasicinfoProcess(page,extrasMap);
            	break;
            case "/artist":
            	this.companySignActorProcess(page,extrasMap);//签约艺人
            	break;	
            default:
                this.addRequestProcess(page);
                break;
        }
		
	}
	
	@Override
	public String getPageDataType(Page page) {
		String url = page.getUrl().toString();
        if (url.contains("/companylist")){
        	return "/companylist";
        }else if (url.contains("/celebritylist")){
			return "/celebritylist";
		}else if(url.contains("/celebrity?id")){
			return "/celebrity?id";
		}else if(url.contains("/companydetail")){
			String tempUrl = url.substring(url.lastIndexOf("/")+1).replaceAll("[0-9]","");
			if(tempUrl.equals("")){
				return "/companydetail";
			}else if(tempUrl.equals("artist")){
				return "/artist";
			}
		}
        return Constants.PageDataType.SEARCH_LIST;
	}

	//添加请求任务
	public void addRequestProcess(Page page){
	
		List<JSONObject> targetList = MaoYanDataSource.jsonList;
		
		System.out.println(targetList);
		
		targetList.forEach(target ->
        {
            String dataId = target.getString("dataid");
            String tvname = target.getString("tvname");
            if (StringUtils.isEmpty(dataId))
                return;
            Map<String, Object> extrasMap = new HashMap<String, Object>();
            extrasMap.put("dataId",dataId);
            extrasMap.put("tvname",tvname);
            PageUtils.addTargetRequest(page, String.format("https://piaofang.maoyan.com/movie/%s/companylist", dataId), extrasMap);
        });
	}
	
	//公司签约艺人
	protected void companySignActorProcess(Page page, Map<String, Object> extrasMap) {
		
		String url = page.getRequest().getUrl();
		
		String companyId = extrasMap.get("companyId").toString();
		
		CompanyBasicinfoModel companyBasicinfoModel = (CompanyBasicinfoModel)extrasMap.get("companyBasicinfoModel");
		
		//签约艺人
		String [] signActor = null;
		
		//签约艺人id
		String [] signActorIds = null;

		//签约艺人职业
		String occupationArr = null;
		
		if(url.contains(companyId)){
			
			
			List<CompanyStarsModel> companyStarsList = new ArrayList<CompanyStarsModel>(); 
			
			List<StarTvWorksModel> starTvWorksList = new ArrayList<StarTvWorksModel>();
			
			List<String> panelCList = page.getHtml().xpath("//div[@class='sticky-container']//div[@class='panel-content']//div[@class='panel-c']").all();
			
			if(null != panelCList && panelCList.size()>0 ){
				signActor = new String[panelCList.size()];
				signActorIds = new String[panelCList.size()];
				for(int i=0;i<panelCList.size();i++){
					Html panelCHtml = new Html(panelCList.get(i));
					signActor[i] = panelCHtml.xpath("//div[@class='p-item']//a[@class='p-link']//p[@class='p-item-name ellipsis-1']//text()").get().trim(); 
					signActorIds[i] = panelCHtml.xpath("//div[@class='p-item']//a[@class='p-link']/@data-id").get();
					
					StarTvWorksModel paramObj =  new StarTvWorksModel();
					paramObj.setStar_dataid(signActorIds[i]);
					starTvWorksList.add(paramObj);
					
					String occupation = panelCHtml.xpath("//div[@class='p-item']//a[@class='p-link']//p[@class='p-item-play ellipsis-1']//text()").get();
					
					if(null != occupation && StringUtils.isNotBlank(occupation)){
						occupationArr = occupation.replaceAll("[\\s+]", "").replaceAll("[|]", ",");
						//occupationArr = occupation.split("#");
					}
					
					//将签约艺人与公司成员放入公司艺人中间表缓存中  并且爬取艺人基本信息
					companyStarsList.add(new CompanyStarsModel(
							UUIDUtils.getId(), null, 
							companyId, companyBasicinfoModel.getCompanyname(), 
							null,signActorIds[i], 
							signActor[i], occupationArr, 
							resourceSiteId, System.currentTimeMillis()
							)); 
				}
				
			}
			
			//存在公司成员
			String[] companyPeoplesArr = companyBasicinfoModel.getCompanypeoples();
			String[] companyPeopleidsArr = companyBasicinfoModel.getCompanypeopleids();
			String[] companyPeoplesroleArr = companyBasicinfoModel.getCompanypeoplesrole();
			
			 
			if(null != companyPeoplesArr && companyPeoplesArr.length>0
					
					&& companyPeoplesArr.length == companyPeopleidsArr.length && 
					
					companyPeopleidsArr.length == companyPeoplesroleArr.length){
				
				for(int i =0;i<companyPeoplesArr.length;i++){
					StarTvWorksModel paramObj =  new StarTvWorksModel();
					paramObj.setStar_dataid(companyPeopleidsArr[i]);
					starTvWorksList.add(paramObj);
					companyStarsList.add(new CompanyStarsModel(
							UUIDUtils.getId(), null, 
							companyId, companyBasicinfoModel.getCompanyname(), 
							null,companyPeopleidsArr[i], 
							companyPeoplesArr[i], 
							(companyPeoplesroleArr[i] == null || StringUtils.isBlank(companyPeoplesroleArr[i]))?null:
								companyPeoplesroleArr[i].replaceAll("[\\s+]", "").replaceAll("[|/,，]", ","), 
							resourceSiteId, System.currentTimeMillis()
							)); 
				}
				
			}
			
			companyBasicinfoModel.setSignactor(signActor);
			companyBasicinfoModel.setSignactorids(signActorIds);
			
			//保存
			page.putField(CompanyStarsModel.class.getName(), companyStarsList);
			
			//检查艺人是否存在艺人基础表中  并添加请求爬取艺人基本信息
			checkExitsStarInfo(page,starTvWorksList);
			
			cacheHandler.hSet(COMPANY_INFO_SUPPLEMENT, companyBasicinfoModel.getCompany_dataid(), companyBasicinfoModel);
			cacheHandler.setCacheTimeout(COMPANY_INFO_SUPPLEMENT);
		}
		
	}


	//公司基本信息
	protected void companyBasicinfoProcess(Page page, Map<String, Object> extrasMap) {
		
		String url = page.getRequest().getUrl();
		
		String companyId = extrasMap.get("companyId").toString();
		
		if(url.contains(companyId)){
			
			//公司图片链接
			String imageUrl = null;
			
			//公司名称
			String companyName = null;
			
			//公司英文名
			String companyEngName = null;
			
			//公司简介
			String introduction = null;
			
			//主要作品
			String [] works = null;
			//主要作品对应的id
			String [] workIds = null;
			
			//公司成员
			String [] companyPeoples = null;
			
			//公司成员角色
			String [] companyPeoplesRole = null;
			
			//公司成员id
			String [] companyPeopleIds = null;
			
						
			//公司性质
			String companyProperty = null;
			
			//公司类型
			String [] companyTypes = null;
			
			//子公司
			String [] childCompanys = null;
			
			//子公司id数组
			String [] childCompanyIds = null;
			
			//母公司名称数组
			String [] parentCompanys = null;
			
			//母公司id数组
			String [] parentCompanyIds = null;
			
			//公司地点
			String address = null;
			
			
			//公司图片
			if(null != page.getHtml().xpath("//section[@class='summary-section']//div[@class='chd-logo']//img/@src")){
				imageUrl = page.getHtml().xpath("//section[@class='summary-section']//div[@class='chd-logo']//img/@src").get();
			}
			
			//公司名称
			if(null != page.getHtml().xpath("//section[@class='summary-section']"
					+ "//div[@class='chd-name-holder']"
					+ "//p[@class='chd-name ellipsis-1']"
					+ "//text()").get()){
				companyName = page.getHtml().xpath("//section[@class='summary-section']"
						+ "//div[@class='chd-name-holder']"
						+ "//p[@class='chd-name ellipsis-1']"
						+ "//text()").get();
			}
			//公司英文名
			if(null != page.getHtml().xpath("//section[@class='summary-section']"
					+ "//div[@class='chd-name-holder']"
					+ "//p[@class='chd-ename ellipsis-1']"
					+ "//text()").get()){
				companyEngName = page.getHtml().xpath("//section[@class='summary-section']"
						+ "//div[@class='chd-name-holder']"
						+ "//p[@class='chd-ename ellipsis-1']"
						+ "//text()").get();
				if(StringUtils.isBlank(companyEngName)){
					companyEngName = null;
				}
			}
			//公司简介
			if(null != page.getHtml().xpath("//section[@class='summary-section']"
					+ "//div[@class='summary-intro']"
					+ "//div[@class='expand-content']//text()").get()){
				introduction = page.getHtml().xpath("//section[@class='summary-section']"
						+ "//div[@class='summary-intro']"
						+ "//div[@class='expand-content']//text()").get();
				if(StringUtils.isBlank(introduction)){
					introduction = null;
				}
			}
			
			//主要作品及作品对应的网站剧id
			List<String> workItemList = page.getHtml().xpath("//section[@class='work-section']//a[@class='work-item']").all();
			if(null != workItemList
					&& workItemList.size()>0){
				works = new String[workItemList.size()];
				workIds = new String[workItemList.size()];
				for(int i=0;i<workItemList.size();i++){
					Html workItemHtml = new Html(workItemList.get(i));
					works[i] = workItemHtml.xpath("//p[@class='wi-name ellipsis-1']//text()").get().trim();
					workIds[i] = workItemHtml.xpath("//a[@class='work-item']/@href").get().replaceAll("[^0-9]", "");
				}
			}
			
			//公司成员及角色及id
			
			if(null!=page.getHtml().xpath("//section[@class='staff-section']//a[@class='staff-item']").all()
					&& page.getHtml().xpath("//section[@class='staff-section']//a[@class='staff-item']").all().size()>0){
				List<String> aTagList = page.getHtml().xpath("//section[@class='staff-section']//a[@class='staff-item']").all();
				String[] propleArr = new String[aTagList.size()];
				String[] roleArr = new String[aTagList.size()];
				companyPeopleIds = new String[aTagList.size()];
				for(int i=0; i<aTagList.size();i++){
					Html aTagHtml = new Html(aTagList.get(i));
					if(null != aTagHtml.xpath("//a[@class='staff-item']//p[@class='si-name ellipsis-1']")){
						if(StringUtils.isNotBlank(aTagHtml.xpath("//a[@class='staff-item']//p[@class='si-name ellipsis-1']//text()").get())){
							propleArr[i] = aTagHtml.xpath("//a[@class='staff-item']//p[@class='si-name ellipsis-1']//text()").get();
						}else{
							propleArr[i] = "";
						}
						if(StringUtils.isNotBlank(aTagHtml.xpath("//a[@class='staff-item']//p[@class='si-role ellipsis-1']//text()").get())){
							roleArr[i] = aTagHtml.xpath("//a[@class='staff-item']//p[@class='si-role ellipsis-1']//text()").get();
						}else{
							roleArr[i] = "";
						}
					}
					companyPeopleIds[i] = aTagHtml.xpath("//a[@class='staff-item']/@href").get().replaceAll("[^0-9]", "");
				}
				companyPeoples = propleArr;
				companyPeoplesRole = roleArr;
				
			}
			
			
			//公司性质
			if(null != page.getHtml().xpath("//section[@class='info-section']"
					+ "//div[@class='expandable theme-text']"
					+ "//div[@class='expand-content']"
					+ "//div[@class='i-item i-nature']"
					+ "//p[@class='i-label']//text()").get()){
				if(page.getHtml().xpath("//section[@class='info-section']"
						+ "//div[@class='expandable theme-text']"
						+ "//div[@class='expand-content']"
						+ "//div[@class='i-item i-nature']"
						+ "//p[@class='i-label']//text()").get().contains("公司性质")){
					companyProperty = page.getHtml().xpath("//section[@class='info-section']"
							+ "//div[@class='expandable theme-text']"
							+ "//div[@class='expand-content']"
							+ "//div[@class='i-item i-nature']"
							+ "//p[@class='i-val ellipsis-1']//text()").get().trim();
					if(StringUtils.isBlank(companyProperty)){
						companyProperty = null;
					}
				}
			}
			
			//公司类型
			if(null != page.getHtml().xpath("//section[@class='info-section']"
					+ "//div[@class='expandable theme-text']"
					+ "//div[@class='expand-content']"
					+ "//div[@class='i-item i-ctype']"
					+ "//p[@class='i-label']//text()").get()){
				if(page.getHtml().xpath("//section[@class='info-section']"
						+ "//div[@class='expandable theme-text']"
						+ "//div[@class='expand-content']"
						+ "//div[@class='i-item i-ctype']"
						+ "//p[@class='i-label']//text()").get().contains("公司类型")){
					companyTypes = page.getHtml().xpath("//section[@class='info-section']"
							+ "//div[@class='expandable theme-text']"
							+ "//div[@class='expand-content']"
							+ "//div[@class='i-item i-ctype']"
							+ "//p[@class='i-val ellipsis-2']//text()").get().replaceAll("[ ]", ",").split(",");
					if(companyTypes.length==0){
						companyTypes = null;
					}
				}
			}
			
			//子公司及子公司id
			List<String> childCompanyList = page.getHtml().xpath("//section[@class='info-section']"
					+ "//div[@class='i-item i-child']"
					+ "//a[@class='i-val ellipsis-1']").all();
			if(null != childCompanyList && childCompanyList.size()>0){
				childCompanys = new String[childCompanyList.size()];
				childCompanyIds = new String[childCompanyList.size()];
				for(int i=0;i<childCompanyList.size();i++){
					Html childCompanyHtml = new Html(childCompanyList.get(i));
					childCompanys[i] = childCompanyHtml.xpath("//span//text()").get().trim();
					childCompanyIds[i] = childCompanyHtml.xpath("//a[@class='i-val ellipsis-1']/@href").get().replaceAll("[^0-9]", "");
				}
			}
			
			//母公司及公司id
			List<String> parentCompanyList = page.getHtml().xpath("//section[@class='info-section']"
					+ "//div[@class='i-item i-parent']"
					+ "//a[@class='i-val ellipsis-1']").all();
			if(null != parentCompanyList && parentCompanyList.size()>0){
				parentCompanys = new String[parentCompanyList.size()];
				parentCompanyIds = new String[parentCompanyList.size()];
				for(int i=0;i<parentCompanyList.size();i++){
					Html parentCompanyHtml = new Html(parentCompanyList.get(i));
					parentCompanys[i] = parentCompanyHtml.xpath("//span//text()").get().trim();
					parentCompanyIds[i] = parentCompanyHtml.xpath("//a[@class='i-val ellipsis-1']/@href").get().replaceAll("[^0-9]", "");
				}
			}
			
			
			//公司地点
			if(null!= page.getHtml().xpath("//section[@class='contact-section']//div[@class='contact']//p[@class='c-address']").get()){
				address = page.getHtml().xpath("//section[@class='contact-section']//div[@class='contact']//p[@class='c-address']//text()").get();
				if(StringUtils.isBlank(address)){
					address = null;
				}
			}
			
			//保存公司的基本信息
			CompanyBasicinfoModel companyBasicinfoModel = new CompanyBasicinfoModel(
					UUIDUtils.getId(),null, 
					companyId, Constants.SiteName.MAOYAN,
					companyName, companyEngName,
					companyProperty, companyTypes,
				    works,workIds, 
				    parentCompanys, parentCompanyIds,
				    childCompanys,childCompanyIds, 
				    address, introduction,
				    companyPeoples,companyPeoplesRole,
				    companyPeopleIds,null, 
				    null,imageUrl, 
				    System.currentTimeMillis()
					);
			
			extrasMap.put("companyBasicinfoModel", companyBasicinfoModel);
			
			//用于提取签约艺人
			PageUtils.addTargetRequest(page, String.format("https://piaofang.maoyan.com/companydetail/%s/artist", companyId), extrasMap);
			
		}
	}


	//艺人基本信息
	protected void starBasicinfoProcess(Page page,Map<String, Object> extrasMap) {
		
		String url = page.getRequest().getUrl();
		
		String starId = extrasMap.get("starId").toString();
		
		if(url.contains(starId)){
			//艺人图片链接
			String imageUrl = page.getHtml().xpath("//div[@class='celebrity-header cele-main']"
					+ "//div[@class='celebrity-bg']"
					+ "//img[@class='sm-big-img']"
					+ "/@src").get();
			//名字
			String starName = null;
			//英文名
			String engName = null;
			List<String> list = page.getHtml().xpath("//div[@class='celebrity-header cele-main']"
					+ "//div[@class='celebrity-bg']"
					+ "//div[@class='celebrity-profile']"
					+ "//div[@class='string-tip']//p//text()").all();
			if(null != list && list.size()>0){
				for(int i =0;i<list.size();i++){
					if(i==0){
						starName = list.get(i).trim();
					}else if(i==1){
						engName = list.get(i).trim();
					}
				}
				
			}
			
			//职业
			String [] occupation = null;
			List<String> occupationList = page.getHtml().xpath("//div[@class='celebrity-header cele-main']"
					+ "//div[@class='celebrity-bg']"
					+ "//div[@class='celebrity-profile']"
					+ "//div[@class='common-area']"
					+ "//div[@class='role-name-list string-ellipsis string-show-all']//span//text()").all();
			if(null!=occupationList && occupationList.size()>0){
				occupation = new String[occupationList.size()];
				for(int i = 0;i<occupationList.size();i++){
					occupation[i] = occupationList.get(i).trim();
				}
			}
			
			//国籍
			String country = null;
			//民族
			String nation = null;
			//性别
			String sex = null;
			//血型
			String bloodType = null;
			//身高
			String height = null;
			//体重
			String weight = null;
			
			
			List<String> contentPageList = page.getHtml().xpath("//div[@class='celeAnchor-container body-container']"
					+ "//div[@class='cele-main-content-area']"
					+ "//div[@class='cele-main cele-desc cele-basic-info expandable']"
					+ "//div[@class='content-page-block']"
					+ "//p[@class='content-page']").all();
			
			for(String content : contentPageList){
				if(null != new Html(content).xpath("//p[@class='content-page']//span").all()){
					if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("国籍")){
						country = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}else if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("民族")){
						nation = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}else if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("性别")){
						sex = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}else if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("身高")){
						height = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}else if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("血型")){
						bloodType = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}else if(new Html(content).xpath("//p[@class='content-page']//span").all().get(1).trim().contains("体重")){
						weight = new Html(content).xpath("//p[@class='content-page']//span//text()").all().get(2).trim();
					}
				};
			}
			//出生日期
			String birthday = null;
			//毕业院校
			String school = null;
			//别名
			String [] alias = null;
			//出生地
			String born = null;
			//简介
			String introduction = null;
			
			List<String> divList = page.getHtml().xpath("//div[@class='content-wrapper']//div[@class='cele-introduction expand-content content-page']//div").all();
			if(null != divList && divList.size()>0){
				for(String divStr : divList){
					Html divHtml = new Html(divStr);
					if(divHtml.xpath("//div/@data-id").toString().contains("birthday")){//1958-07-14（巨蟹座）
						birthday = divHtml.xpath("//div//span//text()").all().get(1).trim().replaceAll("[\u4e00-\u9fa5（）()]", "");
					}else if(divHtml.xpath("//div/@data-id").toString().contains("school")){
						school = divHtml.xpath("//div//span//text()").all().get(1).trim();
					}else if(divHtml.xpath("//div/@data-id").toString().contains("aliasName")){
						alias = divHtml.xpath("//div//span//text()").all().get(1).trim().replaceAll("[\"]", "").replaceAll("[,，/]", ",").split(",");
					}else if(divHtml.xpath("//div/@data-id").toString().contains("birthPlace")){
						born = divHtml.xpath("//div//span//text()").all().get(1).trim();
					}else if(divHtml.xpath("//div/@data-id").toString().contains("intro")){
						introduction = divHtml.xpath("//div//span//text()").all().get(1).trim();
					}
				}
			}
			
			
			StarBasicinfoModel starBasicinfoModel = new StarBasicinfoModel(
					UUIDUtils.getId(),null,starId,Constants.SiteName.MAOYAN,
					starName,engName,occupation,country,
					alias,sex,height,weight,
					bloodType,nation,school,
					born,birthday,introduction,
					imageUrl,System.currentTimeMillis()
					);
			
			cacheHandler.hSet(STAR_INFO_SUPPLEMENT, starId, starBasicinfoModel);
			cacheHandler.setCacheTimeout(STAR_INFO_SUPPLEMENT);
		}
		
	}


	//人员相关
	@SuppressWarnings("unchecked")
	protected void basicinfoPeoplelistProcess(Page page, Map<String, Object> extrasMap) {
		String url = page.getRequest().getUrl();
		String dataid = extrasMap.get("dataId").toString();
		String tvname = extrasMap.get("tvname").toString();
		if(url.contains(dataid)){
			
			//存放艺人与作品
			List<StarTvWorksModel> starTvWorksModelList = new ArrayList<StarTvWorksModel>();
			
			List<String> dlTagAll = page.getHtml().xpath("//div[@class='sticky-container']//div[@class='scroller cat-wrapper']//div[@class='wrapper cat-content']"
					+ "//div[@class='panel-wrapper']//dl").all();
			for(String dl : dlTagAll){
				//获取所有人员
				Map<String,Object> resultAll = convertPeopleRelateInfo(new Html(dl),true,dataid,tvname,starTvWorksModelList);
				starTvWorksModelList = (List<StarTvWorksModel>)resultAll.get("starTvWorksModelList"); 
			}
			
			//检查艺人是否存在艺人基础表中  并添加请求爬取艺人基本信息
			checkExitsStarInfo(page,starTvWorksModelList);
			//保存
			page.putField(StarTvWorksModel.class.getName(), starTvWorksModelList);
		}
	}
	
	
	//检查艺人是否存在艺人基础表中
	private void checkExitsStarInfo(Page page,List<StarTvWorksModel> starTvWorksModelList){
		//艺人去重
		Map<String,String> starDistinctMap = new HashMap<String,String>();
		
		List<StarTvWorksModel> list = new ArrayList<StarTvWorksModel>();
		list.addAll(starTvWorksModelList);
		for(StarTvWorksModel noExitsModel : list){
			if(!starDistinctMap.containsKey(noExitsModel.getStar_dataid())){
				starDistinctMap.put(noExitsModel.getStar_dataid(), noExitsModel.getStarname());
			}
		}
		
		for(Entry<String, String> entry : starDistinctMap.entrySet()){
			Map<String,Object> extrasMap = new HashMap<String,Object>();
			extrasMap.put("starId", entry.getKey());
			if(!cacheHandler.cacheExist(SPIDER_DISTINCT_STAR_INFO)){
				PageUtils.addTargetRequest(page, "https://piaofang.maoyan.com/celebrity?id="+entry.getKey(), extrasMap);
				cacheHandler.hSet(SPIDER_DISTINCT_STAR_INFO, entry.getKey(), resourceSiteId+"_"+entry.getKey()+"_"+entry.getValue());
			}else{
				String hGetAll = cacheHandler.hGet(SPIDER_DISTINCT_STAR_INFO, entry.getKey(), String.class);
				if(null == hGetAll){
					PageUtils.addTargetRequest(page, "https://piaofang.maoyan.com/celebrity?id="+entry.getKey(), extrasMap);
	  	    		cacheHandler.hSet(SPIDER_DISTINCT_STAR_INFO, entry.getKey(), resourceSiteId+"_"+entry.getKey()+"_"+entry.getValue());
	  	    	}
			}
			
		}
		//设置失效时间
		cacheHandler.setCacheTimeout(SPIDER_DISTINCT_STAR_INFO);
	}
	
	
	
	
	private Map<String,Object> convertPeopleRelateInfo(Html dlHtml,boolean isAll,String dataid,String tvname,List<StarTvWorksModel> starTvWorksModelList){
		
		
		Map<String,Object> result = new HashMap<String,Object>();
		
		String occupation = dlHtml.xpath("//dl[@class='panel-main category']//dt[@class='panel-header sticky-wrapper']//span[@class='title-name']//text()").get();
		
		List<String> contentAll = dlHtml.xpath("//dl[@class='panel-main category']//dd[@class='panel-content']//div[@class='p-item']").all();
		String[][] nameAndEngNameAndRole = new String[3][];
		int size = 0;
		if(!isAll){
			size = contentAll.size()<=10?contentAll.size():10;
		}else{
			size = contentAll.size();
		}
		String [] nameArr = new String[size]; 
		String [] engNameArr = new String[size];
		String [] roleArr = new String[size];
//		if(occupation.equals("导演") || occupation.equals("演员") || occupation.equals("编剧")
//				|| occupation.equals("制片人") || occupation.equals("摄影师")
//				|| occupation.equals("出品人") || occupation.equals("发行")){
		int ordertype = 1;
		for(int i=0;i<size;i++){
			
			
			String starId = new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']/@data-id").get();
			
			String name = new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
					+ "//p[@class='p-item-name ellipsis-1']//text()").get()==null?"":new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
							+ "//p[@class='p-item-name ellipsis-1']//text()").get();
			String engName = new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
					+ "//p[@class='p-item-e-name ellipsis-1']//text()").get()==null?"":new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
							+ "//p[@class='p-item-e-name ellipsis-1']//text()").get();
			String role = new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
					+ "//p[@class='p-item-play ellipsis-1']//text()").get()==null?"":new Html(contentAll.get(i)).xpath("//div[@class='p-item']//a[@class='p-link']//div[@class='p-desc']"
							+ "//p[@class='p-item-play ellipsis-1']//text()").get().replace("饰：", "").replace("（配音）", "");
			nameArr[i] = name.replaceAll("[\"]", "").replace(",", "/");
			engNameArr[i] = engName.replaceAll("[\"]", "").replace(",", "/");
			roleArr[i] = role.replaceAll("[\"]", "").replace(",", "/");
			if(isAll){
				starTvWorksModelList.add(new StarTvWorksModel(
						UUIDUtils.getId(),null,null,
						dataid,
						tvname,
						starId, name, role,
						occupation,Constants.SiteName.MAOYAN,
						ordertype++,
						System.currentTimeMillis()
						));
			}
		}
		for(int j=0;j<nameArr.length;j++){
			if(StringUtils.isNotEmpty(nameArr[j])){
				nameAndEngNameAndRole[0] = nameArr;
			}
			if(StringUtils.isNotEmpty(engNameArr[j])){
				nameAndEngNameAndRole[1] = engNameArr;
			}
			if(StringUtils.isNotEmpty(roleArr[j])){
				nameAndEngNameAndRole[2] = roleArr;
			}
		}
			
		//}
		result.put("nameAndEngNameAndRole", nameAndEngNameAndRole);
		result.put("flag", occupation);
		if(null != starTvWorksModelList && isAll){
			result.put("starTvWorksModelList", starTvWorksModelList);
		}
		return result;
	}
	

	//参与的公司
	@SuppressWarnings("unchecked")
	protected void basicinfoCompanylistProcess(Page page, Map<String, Object> extrasMap) {
		String url = page.getRequest().getUrl();
		String dataid = extrasMap.get("dataId").toString();
		String tvname = extrasMap.get("tvname").toString();
		if(url.contains(dataid)){
			//公司
			String [] companyArr = null;
			
			Html html = page.getHtml();
			
			try {
				List<String> all = html.xpath("//div[@class='wrapper cat-content']//div[@class='panel-wrapper']//dl").all();
				List<CompanyTvWorksModel> companyTvWorksModelList = new ArrayList<CompanyTvWorksModel>();
				for(String dlStr : all){
					Html dlHtml = new Html(dlStr);
					String companyNameStr = dlHtml.xpath("//dl[@class='panel-main category']//span[@class='title-left']//span[@class='title-name']//text()").get();
					if(StringUtils.isNotBlank(companyNameStr)){
						String currCompanyNameStr = companyNameStr;
						if(companyNameStr.contains("其他")){
							currCompanyNameStr = "";
						}
						companyTvWorksModelList = (List<CompanyTvWorksModel>)this.getCompanyInfoList(dataid,tvname,dlHtml,currCompanyNameStr,companyArr,true,companyTvWorksModelList).get("companyTvWorksModelList");
					}
				}
				//保存公司代表作
				page.putField(CompanyTvWorksModel.class.getName(), companyTvWorksModelList);
				
				//去重公司
				checkExitsCompanyInfo(page,companyTvWorksModelList);
				
			} catch (Exception e) {
				System.out.println("异常信息 "+MaoYanMovieStarConpanyPageProcessor.class.getName()+" methodName : basicinfoCompanylistProcess "+e.getMessage());
			}
			PageUtils.addTargetRequest(page, String.format("https://piaofang.maoyan.com/movie/%s/celebritylist", dataid), extrasMap);
		}
		
	}

	//检查公司是否存在表中
	private void checkExitsCompanyInfo(Page page, List<CompanyTvWorksModel> companyTvWorksModelList) {
		//公司去重
		Map<String,String> distinctMap = new HashMap<String,String>();
		List<CompanyTvWorksModel> list = new ArrayList<CompanyTvWorksModel>();
		list.addAll(companyTvWorksModelList);
		
		for(CompanyTvWorksModel noExitsModel : list){
			if(!distinctMap.containsKey(noExitsModel.getCompany_dataid())){
				distinctMap.put(noExitsModel.getCompany_dataid(), noExitsModel.getCompanyname());
			}
		}
		
		for(Entry<String, String> entry : distinctMap.entrySet()){
			Map<String,Object> extrasMap = new HashMap<String,Object>();
			extrasMap.put("companyId", entry.getKey());
			if(!cacheHandler.cacheExist(SPIDER_DISTINCT_COMPANY_INFO)){
				cacheHandler.hSet(SPIDER_DISTINCT_COMPANY_INFO, entry.getKey(), resourceSiteId+"_"+entry.getKey()+"_"+entry.getValue());
				PageUtils.addTargetRequest(page, "https://piaofang.maoyan.com/companydetail/"+entry.getKey(), extrasMap);
			}else{
				String hGetAll = cacheHandler.hGet(SPIDER_DISTINCT_COMPANY_INFO, entry.getKey(), String.class);
				if(null == hGetAll){
					PageUtils.addTargetRequest(page, "https://piaofang.maoyan.com/companydetail/"+entry.getKey(), extrasMap);
					cacheHandler.hSet(SPIDER_DISTINCT_COMPANY_INFO, entry.getKey(), resourceSiteId+"_"+entry.getKey()+"_"+entry.getValue());
	  	    	}
			}
		}
		//设置失效时间
		cacheHandler.setCacheTimeout(SPIDER_DISTINCT_COMPANY_INFO);
	}


	private Map<String,Object> getCompanyInfoList(String dataid,String tvname,
			Html html,String companyType,String [] companyArr,
			boolean isAll,List<CompanyTvWorksModel> companyTvWorksModelList){
		List<String> companyList = html.xpath("//dl[@class='panel-main category']//dd[@class='panel-content']"
				+ "//div[@class='panel-c']//div[@class='p-item']"
				+ "//div[@class='p-desc']//p//text()").all();
		List<String> companyIdList = html.xpath("//dl[@class='panel-main category']//dd[@class='panel-content']"
				+ "//div[@class='panel-c']//div[@class='p-item']"
				+ "//div[@class='p-link']/@data-id").all();
		if(companyList.size()>0 && companyIdList.size()>0 && (companyList.size() == companyIdList.size())){
			return convertListToArray(dataid,tvname,companyList,companyIdList,companyType,isAll,companyTvWorksModelList);
		}
		return null;
	}
	
	private Map<String,Object> convertListToArray(String dataid,String tvname,
			List<String> nameList,List<String> idlist,
			String companyType,boolean isAll,List<CompanyTvWorksModel> companyTvWorksModelList){
		Map<String,Object> returnMap = new HashMap<String,Object>();
		String [] tempArr = new String[nameList.size()];
		int ordertype = 1;
		for(int i=0;i<nameList.size();i++){
			tempArr[i] = nameList.get(i).trim();
			if(companyType.equals("need_remove_tail_brackets")){//去除尾部括号
				tempArr[i] = StringUtil.removeBrackets(nameList.get(i).trim());
			}
			if(isAll && null!=companyTvWorksModelList){
				CompanyTvWorksModel companyTvWorksModel = new CompanyTvWorksModel(
						UUIDUtils.getId(),null,null,
						dataid,
						tvname,
						idlist.get(i).trim(),
						companyType.equals("")==true?StringUtil.getBracketsContent(tempArr[i])[0]:tempArr[i],
						companyType.equals("")==true?StringUtil.getBracketsContent(tempArr[i])[1]:companyType,
						Constants.SiteName.MAOYAN,
						ordertype++,
						System.currentTimeMillis()
						);
				
				companyTvWorksModelList.add(companyTvWorksModel);
			}
		}
		returnMap.put("companyTvWorksModelList", companyTvWorksModelList);
		returnMap.put("tempArr", tempArr.length==0?null:tempArr);
		return returnMap;
	}
	
	@Override
	protected String[] getStartRequestUrls() {
		return new String[]{START_URL[0]};
	}
	
	@Override
	protected String getSpiderType() {
		return Constants.SpiderTaskType.MOVIE_BOXOFFICE;
	}

	
	protected String convertDateStr(String str){
    	String first = str.substring(0,4);
    	String second = str.substring(4, 6);
    	String three = str.substring(6);
    	return first+"-"+second+"-"+three;
    }
	
}
